from shared import graph, stats_utils
from src_draft.utils import LOW_IMP_FEATURES
import shared.ml_config_core as ml_config_core
import pandas as pd
from shared.ml_config_core import ModelConfigsCollection
from shared.ml_config_runner import run_tuning_for_configs_collection
from shared.definitions import TuningResult
import numpy as np
import statsmodels.api as sm
from sklearn.metrics import roc_auc_score, accuracy_score, log_loss
from pandas import CategoricalDtype
from Draft import feature_builder_v2

import importlib
from matplotlib import pyplot as plt
import src_draft.utils as shared_utils
import seaborn as sns

C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\sklearn\metrics\_scorer.py:548: FutureWarning: The `needs_threshold` and `needs_proba` parameter are deprecated in version 1.4 and will be removed in 1.6. You can either let `response_method` be `None` or set it to `predict` to preserve the same behaviour.
  warnings.warn(

importlib.reload(shared_utils)

shared_utils.pandas_config(pd)
shared_utils.plt_config(plt)

sns.set_theme(style="darkgrid", palette="pastel")
plt.style.use("fivethirtyeight")

features_matrix = feature_builder_v2.load_datasets_and_prepare_features(drop_meta_data=True,
                                                                        ds_type=feature_builder_v2.DatasetType.FULL)

C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\featuretools\entityset\entityset.py:1914: UserWarning: index SK_BUREAU_ID not found in dataframe, creating new integer column
  warnings.warn(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\woodwork\type_sys\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.
  pd.to_datetime(
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\featuretools\computational_backends\feature_set_calculator.py:828: FutureWarning: The provided callable <function min at 0x0000019C5C314400> is currently using SeriesGroupBy.min. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string "min" instead.
  ).agg(to_agg)
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\featuretools\computational_backends\feature_set_calculator.py:828: FutureWarning: The provided callable <function mean at 0x0000019C5C314CC0> is currently using SeriesGroupBy.mean. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string "mean" instead.
  ).agg(to_agg)
C:\Users\Paulius\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\featuretools\computational_backends\feature_set_calculator.py:828: FutureWarning: The provided callable <function max at 0x0000019C5C3142C0> is currently using SeriesGroupBy.max. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string "max" instead.
  ).agg(to_agg)

Appending previous history
Full DS size: 307511

conditions = [
    features_matrix["PrevRatioRejectedAccepted"].isna(),
    features_matrix["PrevRatioRejectedAccepted"] == 0,
    features_matrix["PrevRatioRejectedAccepted"] <= 0.25,
    features_matrix["PrevRatioRejectedAccepted"] > 0.25
]

conditions_2 = [
    features_matrix["PrevRatioRejectedAccepted"].isna(),
    features_matrix["PrevRatioRejectedAccepted"] == 0,
    features_matrix["PrevRatioRejectedAccepted"] > 0,
]


choices = ["No Previous App.", 'All Accepted', "< 25% Rejected", "> 25% Rejected"]
choices_2 = ["No Previous App.", 'All Accepted', "> 0% Rejected"]
# choices = ['All Accepted', "> 0 Rejected"]
# choices = ['No Previous', '0', '> 0']

features_matrix["PrevRatioRejectedAccepted_cats"] = np.select(conditions, choices, default='No Previous App')
features_matrix["PrevRatioRejectedAccepted_cats_2"] = np.select(conditions, choices, default='No Previous App')

features_matrix["PrevRatioRejectedAccepted_cats"] = features_matrix["PrevRatioRejectedAccepted_cats"].astype("category")
features_matrix["PrevRatioRejectedAccepted_cats_2"] = features_matrix["PrevRatioRejectedAccepted_cats_2"].astype("category")

stats_utils.nan_summary(features_matrix[["PrevRatioRejectedAccepted"]])

add_features = ["PrevRatioRejectedAccepted_cats", "PrevRatioRejectedAccepted_cats_2", "TARGET"]
features_matrix_only_high_imp = features_matrix[shared_utils.HIGH_IMP_FEATURES + add_features]
features_matrix_any_imp = features_matrix[shared_utils.ANY_IMP_FEATURES + add_features]

# TODO impute missing values, either 
stats_utils.nan_summary(features_matrix_only_high_imp)

# TODO impute missing values (mean for numerical, proportion sampling for cat)
# OR inside correlation check just drop rows with missing values for tested columns
importlib.reload(graph)

features_matrix_any_imp_no_nan = features_matrix_only_high_imp.dropna(axis=0, how="any")
features_matrix_any_imp_no_nan = features_matrix_any_imp_no_nan.apply(
    lambda col: col.astype(float) if col.dtype == 'Float64' else col.astype(int) if col.dtype == 'Int64' else col)

graph.render_corr_matrix_based_on_type(features_matrix_any_imp_no_nan)

V:\projects\ppuodz-ML.4.1\shared\graph.py:1269: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.
  corr = round(corr.applymap(pd.to_numeric), 2)

correlation_results = []
for col in features_matrix_any_imp_no_nan.columns:
    if col == "TARGET":
        continue
    x = features_matrix_any_imp_no_nan["TARGET"]
    y = features_matrix_any_imp_no_nan[col]

    corr_value, p_value = stats_utils.correlation_test(x, y)
    
    if p_value < 0.05:
        correlation_results.append({'Column': col, 'Coefficient': corr_value, 'P-Value': p_value})

correlation_df = pd.DataFrame(correlation_results).set_index('Column')
correlation_df = correlation_df.loc[correlation_df['Coefficient'].abs().sort_values(ascending=False).index]
correlation_df.round(3)

features_matrix_only_imp_cat_cols = features_matrix_only_high_imp.select_dtypes(include='category').columns
features_matrix_target_cat = features_matrix_only_high_imp.copy()
features_matrix_target_cat["TARGET"] = features_matrix_target_cat["TARGET"].map(
    lambda x: "Default/Loan With Issues" if x == 1 else "No Issues")

features_matrix_target_cat["PrevRatioRejectedAccepted_cats"].dtype

CategoricalDtype(categories=['< 25% Rejected', '> 25% Rejected', 'All Accepted', 'No Previous App.'], ordered=False, categories_dtype=object)

importlib.reload(graph)
graph.draw_distribution_pie_charts(
    features_matrix_target_cat,
    split_var="TARGET",
    include_cols=features_matrix_only_imp_cat_cols,
    title="Distribution of Categorical Variables Relative to Loan Risk",
    clean_tick_label = False,
)

features_matrix_with_bins = features_matrix_only_high_imp.copy()
numerical_cols = features_matrix_only_high_imp.select_dtypes(
    include=["int64", "float64", "Int64"]
).columns

for col in numerical_cols:
    if features_matrix_with_bins[col].nunique() < 5:
        features_matrix_with_bins[f"{col}_binned"] = features_matrix_with_bins[col].astype("category")
    else:
        features_matrix_with_bins[f"{col}_binned"] = stats_utils.bin_and_label(
            features_matrix_with_bins[col], num_bins=4
        )
    features_matrix_with_bins[col] = features_matrix_with_bins[col]
import numpy as np


conditions = [
    features_matrix["TotalDefaults"] == 0,
    features_matrix["TotalDefaults"] >= 1,
    # features_matrix["TotalDefaults"] > 1
]

choices = ["No Defaults", '1 Defaulted Loans']  #,"> 1 defaulted loan"]
# choices = ['All Accepted', "> 0 Rejected"]
# choices = ['No Previous', '0', '> 0']

features_matrix_with_bins["TotalDefaults_cats"] = np.select(conditions, choices, default='WTF?').astype("object")

features_matrix_with_bins["Defaulted"] = features_matrix_with_bins["TARGET"].map(lambda x: "Yes" if x == 1 else "No")
features_matrix_with_bins.drop(columns=["TARGET", "TARGET_binned"], inplace=True)

features_matrix_with_bins["PrevRatioRejectedAccepted_cats"].dtype

CategoricalDtype(categories=['< 25% Rejected', '> 25% Rejected', 'All Accepted', 'No Previous App.'], ordered=False, categories_dtype=object)

importlib.reload(graph)
for target_y in ["ExtSource2", "AmtCredit", "DaysEmployed"]:
    for c in features_matrix_with_bins.columns:
        if pd.api.types.is_numeric_dtype(features_matrix_with_bins[c]):
            continue

        if target_y in c and "binned" in c:
            continue
            
        if "ExtSource" in target_y and "ExtSource" in c:
            continue
            
        # Drop cols with to many categories
        if features_matrix_with_bins[c].nunique() > 10:
            continue
        # if VERBOSE:
        #     display(f"{c} vs {target_y}")
        res = graph.boxen_plot_by_cat(c, features_matrix_with_bins, target_y)
        
        if res:
            display(res)

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

V:\projects\ppuodz-ML.4.1\shared\graph.py:1470: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  grouped = _df.groupby(c)[y_target]
V:\projects\ppuodz-ML.4.1\shared\graph.py:1483: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  group_counts = _df.groupby(c).size()

import numpy as np
import statsmodels.api as sm
from sklearn.metrics import roc_auc_score, accuracy_score, log_loss

# Plot setup
plt.figure(figsize=(12, 6))

line_styles = ['--', ':', '-.']

x_range = np.linspace(features_matrix[['ExtSource1', 'ExtSource2', 'ExtSource3']].min().min(),
                      features_matrix[['ExtSource1', 'ExtSource2', 'ExtSource3']].max().max(), 100)

# Initialize lists for storing predictions
predictions = {}

colors = plt.cm.get_cmap('tab10', 4)

for i, source in enumerate(['ExtSource1', 'ExtSource2', 'ExtSource3']):
    subset = features_matrix[[source, 'TARGET']].dropna()
    X = sm.add_constant(subset[source])
    y = subset['TARGET']

    model = sm.Logit(y, X).fit(disp=0)
    X_pred = pd.DataFrame({'const': 1, source: x_range})

    y_pred = model.predict(X_pred)
    predictions[source] = y_pred

    plt.plot(x_range, y_pred, color=colors(i), linestyle=line_styles[i], alpha=0.5, label=f'{source} (individual)')

combined_features = features_matrix[['ExtSource1', 'ExtSource2', 'ExtSource3', 'TARGET']].dropna()
X_combined = sm.add_constant(combined_features[['ExtSource1', 'ExtSource2', 'ExtSource3']])
y_combined = combined_features['TARGET']

model_combined = sm.Logit(y_combined, X_combined).fit(disp=0)
X_pred_combined = pd.DataFrame({'const': 1, 'ExtSource1': x_range, 'ExtSource2': x_range, 'ExtSource3': x_range})

y_pred_combined = model_combined.predict(X_pred_combined)
y_pred_combined_for_metrics = model_combined.predict(X_combined)

predictions['Combined'] = y_pred_combined

residuals_combined = y_combined - model_combined.predict(X_combined)
residual_std_combined = np.std(residuals_combined)

combined_color = colors(3)  # Selecting the fourth color for the combined model
plt.plot(x_range, y_pred_combined, color=combined_color,
         label='Combined - Predicted Default Probability')  # Solid line for combined model
plt.fill_between(x_range, y_pred_combined - residual_std_combined, y_pred_combined + residual_std_combined,
                 color=combined_color, alpha=0.2)

auc_combined = roc_auc_score(y_combined, y_pred_combined_for_metrics)
accuracy_combined = accuracy_score(y_combined, y_pred_combined_for_metrics.round())  # Assuming binary classification
logloss_combined = log_loss(y_combined, y_pred_combined_for_metrics)

metrics = f"AUC: {auc_combined:.2f}, Accuracy: {accuracy_combined:.2f}, Log-loss: {logloss_combined:.2f}"
plt.annotate(metrics, xy=(0.01, -0.175), xycoords='axes fraction', fontsize=14, color='black')

plt.title('Predicted Probability of Default by Credit Score Source\n(Logit)')
plt.xlabel('Normalized Credit Score')
plt.ylabel('Probability of Default')
plt.legend()
plt.show()

C:\Users\Paulius\AppData\Local\Temp\ipykernel_35252\2151574185.py:16: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  colors = plt.cm.get_cmap('tab10', 4)

model_params = model_combined.params
p_values = model_combined.pvalues
conf_int = model_combined.conf_int()
std_errors = model_combined.bse

coeff_df = pd.DataFrame({
    'Coefficient': model_params,
    'Standard Error': std_errors,
    'P-Value': p_values,
    'Conf. Interval Lower': conf_int[0],
    'Conf. Interval Upper': conf_int[1]
})
coeff_df.round(3)

# Plotting
plt.figure(figsize=(12, 6))
for i in range(1, 2):
    col = f'ExtSource{i}'
    sns.kdeplot(data=features_matrix[features_matrix['TARGET'] == 1][col], label=f'{col} - Default', shade=True)
    sns.kdeplot(data=features_matrix[features_matrix['TARGET'] == 0][col], label=f'{col} - No Default', shade=True)

plt.title('Density Plot of ExtSource Scores by Default Status')
plt.xlabel('Normalized Credit Score')
plt.ylabel('Density')
plt.legend()
plt.show()

C:\Users\Paulius\AppData\Local\Temp\ipykernel_35252\3397180186.py:5: FutureWarning: 

`shade` is now deprecated in favor of `fill`; setting `fill=True`.
This will become an error in seaborn v0.14.0; please update your code.

  sns.kdeplot(data=features_matrix[features_matrix['TARGET'] == 1][col], label=f'{col} - Default', shade=True)
C:\Users\Paulius\AppData\Local\Temp\ipykernel_35252\3397180186.py:6: FutureWarning: 

`shade` is now deprecated in favor of `fill`; setting `fill=True`.
This will become an error in seaborn v0.14.0; please update your code.

  sns.kdeplot(data=features_matrix[features_matrix['TARGET'] == 0][col], label=f'{col} - No Default', shade=True)

from scipy.stats import gaussian_kde
import pandas as pd
import numpy as np
import statsmodels.api as sm
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import roc_auc_score

df = features_matrix[['ExtSource1', 'ExtSource2', 'ExtSource3', "TARGET", "AmtCredit"]].copy()

# Calculate the average of all ExtSources
df['ExtSourceAvg'] = df[['ExtSource1', 'ExtSource2', 'ExtSource3']].mean(axis=1, skipna=True)

# sources = ['ExtSource1']#, 'ExtSource2', 'ExtSource3', 'ExtSourceAvg']
sources = ['ExtSource1', 'ExtSource2', 'ExtSource3', 'ExtSourceAvg', 'AmtCredit']

for source in sources:
    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(16, 6))
    fig.suptitle(f'Analysis for {source}', fontsize=16, y=1.05)  # Top-level title

    # Separate the data
    subset_default = df[df['TARGET'] == 1][source].dropna()
    subset_non_default = df[df['TARGET'] == 0][source].dropna()

    # Total number of observations with valid data
    total_count = len(df[source].dropna())

    # Define the range for the KDE
    score_range = np.linspace(df[source].min(), df[source].max(), 300)

    # KDE for defaults
    kde_default = gaussian_kde(subset_default, bw_method='silverman')
    density_default = kde_default(score_range) * len(subset_default) / total_count

    # KDE for non-defaults
    kde_non_default = gaussian_kde(subset_non_default, bw_method='silverman')
    density_non_default = kde_non_default(score_range) * len(subset_non_default) / total_count

    # Plotting
    # TODO: add fill with alpha like kde plots
    sns.lineplot(x=score_range, y=density_default, ax=ax1, label='Default Probability')
    sns.lineplot(x=score_range, y=density_non_default, ax=ax1, label='Non Default Probability')

    ax1.set_title(f'KDE', fontsize=10)  # Smaller font size for subplot title
    ax1.set_xlabel('Normalized Credit Score')
    ax1.set_ylabel('Density')
    ax1.legend()

    # Regression Plot
    subset = df[[source, 'TARGET']].dropna()

    sns.kdeplot(
        data=subset,
        x=source,
        hue="TARGET",
        # kind="kde",
        # height=6,
        multiple="fill",
        ax=ax2
        # clip=(10, 80),
    )
    # plt.title("Default Rate and EXT_SOURCE_1", x=0.5, y=1.025, fontdict={"size": 16})

    ax2.set_xlabel('Normalized Credit Score')
    ax2.set_ylabel('Probability of Default')
    # ax2.legend()

    # ROC AUC as annotation
    # roc_auc = roc_auc_score(y, model.predict(X))
    # ax2.annotate(f'ROC AUC: {roc_auc:.2f}', xy=(0.05, 0.95), xycoords='axes fraction', fontsize=12, verticalalignment='top')

    plt.tight_layout()
    plt.show()

features_matrix_with_bins["PrevRatioRejectedAccepted_cats"].value_counts()

PrevRatioRejectedAccepted_cats
All Accepted        190370
> 25% Rejected       66215
< 25% Rejected       34079
No Previous App.     16847
Name: count, dtype: int64

features_matrix_with_bins["TotalDefaults_cats"].value_counts()

TotalDefaults_cats
No Defaults          304114
1 Defaulted Loans      3397
Name: count, dtype: int64

features_matrix_with_bins["TotalDefaults"].value_counts()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\pandas\core\indexes\base.py:3805, in Index.get_loc(self, key)
   3804 try:
-> 3805     return self._engine.get_loc(casted_key)
   3806 except KeyError as err:

File index.pyx:167, in pandas._libs.index.IndexEngine.get_loc()

File index.pyx:196, in pandas._libs.index.IndexEngine.get_loc()

File pandas\\_libs\\hashtable_class_helper.pxi:7081, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\\_libs\\hashtable_class_helper.pxi:7089, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'TotalDefaults'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[22], line 1
----> 1 features_matrix_with_bins["TotalDefaults"].value_counts()

File ~\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\pandas\core\frame.py:4102, in DataFrame.__getitem__(self, key)
   4100 if self.columns.nlevels > 1:
   4101     return self._getitem_multilevel(key)
-> 4102 indexer = self.columns.get_loc(key)
   4103 if is_integer(indexer):
   4104     indexer = [indexer]

File ~\AppData\Local\pypoetry\Cache\virtualenvs\ppuodz-ml-4-1-dqELbViF-py3.12\Lib\site-packages\pandas\core\indexes\base.py:3812, in Index.get_loc(self, key)
   3807     if isinstance(casted_key, slice) or (
   3808         isinstance(casted_key, abc.Iterable)
   3809         and any(isinstance(x, slice) for x in casted_key)
   3810     ):
   3811         raise InvalidIndexError(key)
-> 3812     raise KeyError(key) from err
   3813 except TypeError:
   3814     # If we have a listlike key, _check_indexing_error will raise
   3815     #  InvalidIndexError. Otherwise we fall through and re-raise
   3816     #  the TypeError.
   3817     self._check_indexing_error(key)

KeyError: 'TotalDefaults'

features_matrix["PrevRatioRejectedAccepted"].describe()

features_matrix_with_bins["TotalDefaults_cats"].value_counts()

features_matrix_with_bins["PrevRatioRejectedAccepted_cats"].value_counts()

features_matrix["TotalDefaults"].describe()

list(features_matrix_with_bins.columns)

shared_utils.ANY_IMP_FEATURES

graph.boxen_plots_by_category(
    source_df=features_matrix_with_bins,
    group_col="pass__purpose",
    target_col="pass__loan_amnt",
    title="Loan Amount by Purpose",
    x_label="Loan Amount",
)

importlib.reload(graph)
for target_y in ["AmtCredit"]:
    for c in ["CodeGender", "DaysEmployed_binned", "NameEducationType", "OccupationType", "OwnCarAge_binned",
              "DaysBirth_binned", "NameFamilyStatus"
        , "NameFamilyStatus"]:
        # for c in features_matrix_with_bins.columns:
        #     if pd.api.types.is_numeric_dtype(features_matrix_with_bins[c]):
        #         continue
        #     
        #     for 
        display(graph.boxen_plot_by_cat(c, features_matrix_with_bins, target_y, drop_small_cats=True))

	Total NaN Values	Proportion NaN (%)
ExtSource2	660	0.0
ExtSource3	60965	20.0
ExtSource1	173378	56.0
AmtGoodsPrice	278	0.0
OwnCarAge	202929	66.0
PrevAmtDownPaymentSum	16454	5.0
AmtAnnuity	12	0.0
MeanbureaudaysCredit	44020	14.0
MeanbureauamtCreditSumDebt	51380	17.0
PrevAvgYieldGroup	18945	6.0
PrevCreditReceivedRequestedDiff	16454	5.0
OccupationType	96391	31.0
PrevRatioRejectedAccepted	16847	5.0
MaxbureaudaysCreditEnddate	46269	15.0
PrevLastLoanGoodsCategory	16454	5.0
MeanbureauamtCreditMaxOverdue	123625	40.0

	Coefficient	P-Value
Column
ExtSource3	-0.161	0.000
ExtSource1	-0.131	0.000
ExtSource2	-0.128	0.000
MeanbureaudaysCredit	0.093	0.000
OccupationType	0.075	0.000
DaysEmployed	0.074	0.000
PrevRatioRejectedAccepted	0.073	0.000
PrevRatioRejectedAccepted_cats	0.072	0.000
PrevRatioRejectedAccepted_cats_2	0.072	0.000
OrganizationType	0.069	0.000
NameEducationType	0.067	0.000
PrevAmtDownPaymentSum	-0.057	0.000
PrevCreditReceivedRequestedDiff	0.055	0.000
DaysBirth	0.053	0.000
PrevLastLoanGoodsCategory	0.051	0.000
OwnCarAge	0.050	0.000
MeanbureauamtCreditSumDebt	0.049	0.000
MeanbureauamtCreditMaxOverdue	0.044	0.000
DaysIdPublish	0.042	0.000
CodeGender	0.041	0.000
PrevAvgYieldGroup	0.040	0.000
FlagDocument3	0.039	0.000
AmtGoodsPrice	-0.034	0.000
MaxbureaudaysCreditEnddate	0.034	0.000
NameFamilyStatus	0.027	0.002
AmtCredit	-0.023	0.001

	Coefficient	Standard Error	Conf. Interval Lower	Conf. Interval Upper
const	0.600	0.040	0.521	0.680
ExtSource1	-2.099	0.061	-2.219	-1.979
ExtSource2	-1.964	0.060	-2.082	-1.846
ExtSource3	-2.779	0.062	-2.902	-2.657

Exploratory Analysis¶

Relationships Between Numerical and Categorical Variables¶

External Credit Scores (`ExtSource1`)¶

Previous Loan History and Default Risk¶

Exploratory Analysis¶

Relationships Between Numerical and Categorical Variables¶

External Credit Scores (ExtSource1)¶

Previous Loan History and Default Risk¶

External Credit Scores (`ExtSource1`)¶